import os
from dataclasses import dataclass
from multiprocessing import Queue, Process


# noinspection PyShadowingNames
def cut_file(filepath, num_cut, skipline=-1):
    """
    切割文件成块
    :param filepath: 文件路径
    :param num_cut: 需要把文件切分的块的数目
    :param skipline: 
    :return: 
    """
    blocks = []
    file_size = os.path.getsize(filepath)
    block_size = file_size // num_cut
    print(f"file size = {file_size}")
    print(f"block number = {num_cut}")
    print(f"block size = {block_size}")

    with open(filepath, 'rb') as f:
        y = f.tell()
        count = 0
        while True:
            x = y
            f.seek(f.tell() + block_size, os.SEEK_SET)
            f.readline()
            y = f.tell()
            blocks.append(Block(start=x, end=y))
            count += 1
            if y >= file_size:
                break
    return blocks


@dataclass
class Block:
    """
    块的定义，前后两个指针
    """
    start: int
    end: int


# noinspection PyShadowingNames
def handle_block(name, block: Block, q: Queue, eq: Queue):
    """
    读取每个块的每一行，进行处理
    :param name:
    :param block:
    :param q:
    :param eq:
    :return:
    """
    try:
        f = open('large.txt', 'rb')
        f.seek(block.start)
        i = 0
        while True:
            i += 1
            line = f.readline().decode(encoding='utf-8')
            if line is None or line == '':
                break
            if f.tell() - block.start > block.end - block.start:
                print(f"Number {name} block scan end...")
                break
            print(f"{name} line{i}-->{line}")
            var = '{' + line + '}'
            data = eval(var)

            result_txt = handle(data['id'], data['symbol'], float(data['price']), float(data['quantity']), data['type'],
                                data['datetime'])
            q.put(result_txt)
    except Exception as e:
        eq.put(f"handle error:{block},errmsg={str(e)}")


def print_list(objs: list):
    for index, value in enumerate(objs):
        print(f"{index}:{value}")


# noinspection PyUnusedLocal,PyShadowingBuiltins
def handle(id: str, symbol: str, price: float, quantity: float, type: str, dt: str) -> str:
    return f"id={id} value={price * quantity}"


def writer2result(q: Queue):
    """

    :param q: 消息队列
    :return: 把处理后的数据写入指定文件
    """
    with open('result.txt', 'w+') as f:
        for ind in range(10000):
            f.write(q.get())
            f.write('\n')


if __name__ == '__main__':
    filepath = "large.txt"
    num_cut = 50  # 分块
    blocks = cut_file(filepath=filepath, num_cut=num_cut)  # 切割成块
    queue = Queue()  # 数据队列
    error_queue = Queue()  # 错误信息队列
    # 工作进程
    buff = list()
    for i, block in enumerate(blocks):
        buff.append(Process(target=handle_block, args=(f"PROCESS [{i}]", block, queue, error_queue)))
    # 消费者进程
    buff.append(Process(target=writer2result, args=(queue,)))
    [p.start() for p in buff]
    [p.join() for p in buff]
